%%%-------------------------------------------------------------------
%%% File    : mega_data.erl
%%% Author  :  <vjache>
%%% Description : Utility to generate a big amount of test data.
%%%
%%% Created : 25 May 2009 by  <vjache>
%%%-------------------------------------------------------------------
-module(mega_data).

%% API
-export([
		 generate_quads/3,
		 generate_quads/2,
		 new/6,
		 gen/1,
		 gen/2,
		 test/1,
		 test/3]).

-include("../include/rdf_record.hrl").
-include("../include/rrs.hrl").

-record(settings,{
				  tot_popul,
				  max_child,
				  max_friends,
				  max_interests,
				  dict_names,
				  dict_fnames,
				  dict_interests,
				  dict_companies,
				  dict_dir,
				  out_dir="./",
				  emploee_fraction=0.4}).

%%====================================================================
%% API
%%====================================================================
%%--------------------------------------------------------------------
%% Function: 
%% Description:
%%--------------------------------------------------------------------
new(MaxPopul,MaxChild,MaxFriends,MaxInterests,DictsDir,OutDir) ->
	NameList=read_lines(DictsDir++"/names.txt"),
	FNameList=read_lines(DictsDir++"/familynames.txt"),
	InterestList=read_lines(DictsDir++"/interests.txt"),
	CompanyIter=iter_term_reader:new_from_file(DictsDir++"/companies.tup"),
	#settings{
			  dict_names=array:from_list(NameList),
			  dict_fnames=array:from_list(FNameList),
			  dict_interests=array:from_list(InterestList),
			  dict_companies=extract_keys(CompanyIter),
			  tot_popul=MaxPopul,
			  max_child=MaxChild,
			  max_friends=MaxFriends,
			  max_interests=MaxInterests,
			  dict_dir=DictsDir,
			  out_dir=OutDir
			 }.

generate_quads(Setts,Pipe) ->
	PipeState=pipe:open(Pipe),
	PipeState1=gen_loop(Setts,PipeState,1,Setts#settings.tot_popul),
	pipe:close(PipeState1).

generate_quads(Setts,Pipe,{Start,End}=_Range) ->
	PipeState=pipe:open(Pipe),
	PipeState1=gen_loop(Setts,PipeState,Start,End),
	pipe:close(PipeState1).

gen(Setts) ->
	{Nodes,_}=svc_rrs:get_nodes(),
	gen(Setts,Nodes).
gen(Setts,Nodes) ->
	Part=Setts#settings.tot_popul div length(Nodes),
	{NodeRange,_}=lists:foldl(fun(N,{R,Start})->
									  End=Start+Part,
									  {[{N,{Start,End-1}}|R],End}
							  end, {[],_LastPos=1}, Nodes),
	Proms=[
		   begin
			   LogName="mega_data_"++atom_to_list(Node),
			   File=Setts#settings.out_dir++"/"++LogName++".LOG",
			   filelib:ensure_dir(File),
			   file:delete(File),
			   Pipe=pipe_disk_logger:new([{name,LogName},{file,File}]),
			   rpc:async_call(
				 Node, 
				 ?MODULE, generate_quads, _Args=[Setts,Pipe,Range])
		   end
		   ||{Node,Range}<-NodeRange],
	[rpc:yield(Prom)||Prom<-Proms].
	
	

gen_loop(_Setts,PipeState,Min,Max) when Min>Max ->
	PipeState;
gen_loop(Setts,PipeState,Min,Max) ->
	QuadList=gen_person_graph({onto,Min},Setts),
	{_,PipeState1}=pipe:push_each(PipeState, iter:from_list(QuadList)),
	erlang:yield(),
	gen_loop(Setts,PipeState1,Min+1,Max).



gen_person_graph(Id,Setts) ->
	gen_person_graph(Id,Setts,27,55,Setts#settings.max_child,auto).

gen_person_graph(Id,Setts,AgeMin,AgeMax,ChildrenMaxNum,FamilyName) ->
	Context={context,Id},
	DNames=Setts#settings.dict_names,
	Name=array:get(random:uniform(array:size(DNames)), DNames),
	FName=case FamilyName of
			  auto ->
				  DFNames=Setts#settings.dict_fnames,
				  array:get(random:uniform(array:size(DFNames)), DFNames);
			  _ ->
				  FamilyName
		  end,
	Age=AgeMin+random:uniform(AgeMax-AgeMin),
	DInterests=Setts#settings.dict_interests,
	Interests=[
			   q(Id,{onto,"hasInterest"},{'$lit',array:get(random:uniform(array:size(DInterests)), DInterests),string},Context)||_<-lists:seq(1,random:uniform(Setts#settings.max_interests))],
	Children=case ChildrenMaxNum of
				 0 ->
					 [];
				 _ ->
					 ChildrenNum=random:uniform(ChildrenMaxNum),
					 {onto,ParId}=Id,
					 [
					  begin
						  ChildId={onto,{ParId,ChId}},
						  [q(Id,{onto,"hasChild"},ChildId,Context),
						   gen_person_graph(ChildId,Setts,0,Age-20,0,FName)]
					  end||ChId<-lists:seq(1,ChildrenNum)
					 ]
			 end,
	Employment=case FamilyName of
				   auto ->
					   Threshold=1000*Setts#settings.emploee_fraction,
					   case random:uniform(1000) of
						   V when V<Threshold ->
							   {onto,PersId}=Id,
							   EmplId={onto,"empl-"++integer_to_list(PersId)},
							   DComps=Setts#settings.dict_companies,
							   CompId=array:get(random:uniform(array:size(DComps)), DComps),
							   Titles={"worker","manager","analyst","researcher","administrator"},
							   Title=element(random:uniform(tuple_size(Titles)),Titles),
							   [q(EmplId,{onto,"who"},Id,Context),
								q(EmplId,{onto,"title"},{'$lit',Title,string},Context),
								q(EmplId,{onto,"where"},CompId,Context)];
						   _ ->
							   []
					   end;
				   _ ->
					   []
			   end,
	Friends=[q(Id,{onto,"hasFriend"},{onto,random:uniform(Setts#settings.tot_popul)},Context)||_<-lists:seq(1,random:uniform(Setts#settings.max_friends))],
	lists:flatten([
				   q(Id,{onto,"hasName"},{'$lit',Name,string},Context),
				   q(Id,{onto,"hasFamilyName"},{'$lit',FName,string},Context),
				   q(Id,{onto,"hasAge"},{'$lit',Age,int},Context),
				   Interests,
				   Children,
				   Friends,
				   Employment
				  ]).

q(S,P,O,G) ->
	{?SPOG_TAG,S,P,O,G}.

%%====================================================================
%% Internal functions
%%====================================================================
extract_keys(TupIter) ->
	{IterState,KeySet}=iter:fold(TupIter, 
			  fun(Tup,AccSet) ->
					  ordsets:add_element(element(1,Tup), AccSet)
			  end, ordsets:new()),
	ok=iter:close(IterState),
	array:from_list(ordsets:to_list(KeySet)).
read_lines(Filename) ->
	{ok,Dev}=file:open(Filename, [read]),
	loop(Dev,[]).
loop(Dev,Lines) ->
	case io:get_line(Dev,in) of
		eof ->
			Lines;
		Line ->
			loop(Dev,[string:strip(string:strip(Line, right, $\n),right,$\t)|Lines])
	end.
%%====================================================================
%% Test functions
%%====================================================================
test(Pop) ->
	Sett=new(Pop,_Children=4,_Fri=5,_Inters=3,"D:/_Development/Erlang/wsp/demetra/test/valg_bulk_SUITE_data","./"),
%% 	generate_quads(Sett,pipe_console:new()).
%% 	generate_quads(Sett,pipe_disk_logger:new([{name,"mega_data"}])).
	gen(Sett).
test(Pop,DictDir,OutDir) ->
	Sett=new(Pop,_Children=4,_Fri=5,_Intersts=3,DictDir,OutDir),
%% 	generate_quads(Sett,pipe_console:new()).
%% 	generate_quads(Sett,pipe_disk_logger:new([{name,"mega_data"}])).
	gen(Sett).

